I. Web Scrapping of DINA - Concytec

1 First step

The ALICIA - Concytec page was used to search for undergraduate theses with the following search strategy:

(“Análisis Factorial” OR “Análisis de Componentes Principales” OR “ACP” OR “confiabilidad” OR “fiabilidad” OR “análisis psicométrico” OR “baremación”) OR ((“validez” OR “validación” OR “adaptación” OR “construcción” OR “estandarización”) AND (“escala” OR “batería de pruebas” OR “prueba psicológica” OR “instrumento” OR “cuestionario” OR “test”))

In addition, the following filters were used:

• Year from 2011 to 2020
• Undergraduate thesis
• Open Access thesis

All this is kept in a static link that will be used to inspect and store the data in an excel file.

library(rvest)
library(tidyverse)

url <- "https://alicia.concytec.gob.pe/vufind/Search/Results?filter%5B%5D=format%3A%22bachelorThesis%22&filter%5B%5D=eu_rights_str_mv%3A%22openAccess%22&lookfor=%28%E2%80%9CAn%C3%A1lisis+Factorial%E2%80%9D+OR+%E2%80%9CAn%C3%A1lisis+de+Componentes+Principales%E2%80%9D+OR+%E2%80%9CACP%E2%80%9D+OR+%E2%80%9Cconfiabilidad%E2%80%9D+OR+%E2%80%9Cfiabilidad%E2%80%9D+OR+%E2%80%9Can%C3%A1lisis+psicom%C3%A9trico%E2%80%9D+OR+%22baremaci%C3%B3n%22%29+OR+%28%28%E2%80%9Cvalidez%E2%80%9D+OR+%22validaci%C3%B3n%22+OR+%22adaptaci%C3%B3n%22+OR+%22construcci%C3%B3n%22+OR+%22estandarizaci%C3%B3n%22%29+AND+%28%22escala%22+OR+%E2%80%9Cbater%C3%ADa+de+pruebas%E2%80%9D+OR+%E2%80%9Cprueba+psicol%C3%B3gica%E2%80%9D+OR+%22instrumento%22+OR+%22cuestionario%22+OR+%22test%22%29%29&type=AllFields&daterange%5B%5D=publishDate&publishDatefrom=2011&publishDateto=2020"

dina_html <- read_html(url) 

1.1 Determinate the number of pages

last_n_page

[1] "811"

1.2 Loop for extract links in every pages avaible

options(timeout= 1e+10)

group_pages <- split(seq_len(last_n_page), ceiling(seq_len(last_n_page)/50))

tmp <- list()
href_dina <- list()

for (i in seq_along(group_pages)) {
  for (j in group_pages[[i]]) {
    tmp[[j]] <- paste0(url, "&page=", j)

    href_dina[[j]] <- tmp[[j]] %>%
      read_html() %>%
      html_elements(".result .row .link a") %>%
      html_attrs() %>%
      unlist() %>%
      as_tibble()

    print(j)
  }
  Sys.sleep(120)
}

complete_href_dina <- href_dina %>% 
  bind_rows() %>% 
  mutate(
    tesis_url = paste0("https://alicia.concytec.gob.pe",
                       value)
  ) %>% 
  select(-value)

complete_href_dina

2 Loop for read all thesis

2.1 Read html of each thesis

read_html_thesis <- vector("list", nrow(complete_href_dina))

for (i in seq_len(nrow(complete_href_dina))) {
  if (!(complete_href_dina$tesis_url[i] %in% names(read_html_thesis))) {
    cat(paste("Doing thesis number", i, "..."))
    ok <- FALSE
    counter <- 0
    while (ok == FALSE & counter <= 10) {
      counter <- counter + 1
      out <- tryCatch(
        expr = {
          complete_href_dina$tesis_url[i] %>% 
            read_html()
        },
        error = function(e) {
          Sys.sleep(2)
          e
        }
      )
      if ("error" %in% class(out)) {
        cat(".")
      } else {
        ok <- TRUE
        cat("Successful!")
      }
    }
    cat("\n")
    read_html_thesis[[i]] <- out
    names(read_html_thesis)[i] <- complete_href_dina$tesis_url[i]
  }
}

2.2 Extract information

This function help to extract information about thesis like title, abstract, etc.

extract_information <- function(html) {
  titulo <- html %>% 
    html_elements(".media-body h1") %>% 
    html_text2() %>% 
    tibble(Titulo = .)
  
  info_ident <- html %>%  
    html_table() %>% 
    magrittr::extract2(1) %>% 
    mutate(X1 = str_remove(X1, ":")) %>% 
    pivot_wider(
      names_from = X1,
      values_from = X2
    )
  
  resumen <- html %>%  
    html_table() %>% 
    magrittr::extract2(2) %>% 
    mutate(X1 = str_remove(X1, ":")) %>% 
    pivot_wider(
      names_from = X1,
      values_from = X2
    )
  
  information <- bind_cols(
    info_ident,
    titulo,
    resumen
  ) %>% 
    relocate(Titulo, .after = "Autor Principal")
  
  return(information)
}

thesis_information <- vector("list", length(read_html_thesis))

for (i in seq_len(length(read_html_thesis))) {
  thesis_information[[i]] <- extract_information(read_html_thesis[[i]])
}

Join full information about the thesis

2.3 Remove duplicates

The table has 16211 at the moment.

thesis_final <- thesis_information %>% 
  mutate(
    titulo_tmp = str_to_upper(Titulo)
  ) %>%
  distinct(titulo_tmp, .keep_all = TRUE) %>% 
  select(-c(titulo_tmp))

Now, it has 15150.

3 Export to XLSX

openxlsx::write.xlsx(thesis_information,
                     "Table complet thesis psychometric.xlsx")